From cde43db53a333890c4bad387876df15f45faeeba Mon Sep 17 00:00:00 2001 From: "kaf24@firebug.cl.cam.ac.uk" Date: Fri, 18 Nov 2005 17:42:45 +0100 Subject: [PATCH] Improve make_page_readonly and make_page_writable: 1. Use update_va_mapping where possible (much faster) 2. If that fails, fall back to explicit pte update rather than using writable pagetables (very slow for singleton updates) Also many x86/64 cleanups and fix to handle vmalloc space correctly. Signed-off-by: Keir Fraser --- .../arch/xen/i386/mm/pgtable.c | 18 ++- .../arch/xen/x86_64/mm/init.c | 103 +++++++----------- 2 files changed, 51 insertions(+), 70 deletions(-) diff --git a/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c b/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c index e7fbddcb05..9182e0708e 100644 --- a/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c +++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c @@ -386,19 +386,26 @@ asmlinkage int xprintk(const char *fmt, ...); void make_lowmem_page_readonly(void *va) { pte_t *pte = virt_to_ptep(va); - set_pte(pte, pte_wrprotect(*pte)); + int rc = HYPERVISOR_update_va_mapping( + (unsigned long)va, pte_wrprotect(*pte), 0); + BUG_ON(rc); } void make_lowmem_page_writable(void *va) { pte_t *pte = virt_to_ptep(va); - set_pte(pte, pte_mkwrite(*pte)); + int rc = HYPERVISOR_update_va_mapping( + (unsigned long)va, pte_mkwrite(*pte), 0); + BUG_ON(rc); } void make_page_readonly(void *va) { pte_t *pte = virt_to_ptep(va); - set_pte(pte, pte_wrprotect(*pte)); + int rc = HYPERVISOR_update_va_mapping( + (unsigned long)va, pte_wrprotect(*pte), 0); + if (rc) /* fallback? */ + xen_l1_entry_update(pte, pte_wrprotect(*pte)); if ((unsigned long)va >= (unsigned long)high_memory) { unsigned long pfn = pte_pfn(*pte); #ifdef CONFIG_HIGHMEM @@ -412,7 +419,10 @@ void make_page_readonly(void *va) void make_page_writable(void *va) { pte_t *pte = virt_to_ptep(va); - set_pte(pte, pte_mkwrite(*pte)); + int rc = HYPERVISOR_update_va_mapping( + (unsigned long)va, pte_mkwrite(*pte), 0); + if (rc) /* fallback? */ + xen_l1_entry_update(pte, pte_mkwrite(*pte)); if ((unsigned long)va >= (unsigned long)high_memory) { unsigned long pfn = pte_pfn(*pte); #ifdef CONFIG_HIGHMEM diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c index a2182d5c98..ff58486f33 100644 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c +++ b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c @@ -55,11 +55,9 @@ extern char _stext[]; DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); extern unsigned long start_pfn; -static int init_mapping_done; - /* * Use this until direct mapping is established, i.e. before __va() is - * avaialble in init_memory_mapping(). + * available in init_memory_mapping(). */ #define addr_to_page(addr, page) \ @@ -68,91 +66,65 @@ static int init_mapping_done; (((mfn_to_pfn((addr) >> PAGE_SHIFT)) << PAGE_SHIFT) + \ __START_KERNEL_map))) -static void __make_page_readonly(unsigned long va) +static void early_make_page_readonly(void *va) { - unsigned long addr; + unsigned long addr, _va = (unsigned long)va; pte_t pte, *ptep; unsigned long *page = (unsigned long *) init_level4_pgt; - addr = (unsigned long) page[pgd_index(va)]; + addr = (unsigned long) page[pgd_index(_va)]; addr_to_page(addr, page); - addr = page[pud_index(va)]; + addr = page[pud_index(_va)]; addr_to_page(addr, page); - addr = page[pmd_index(va)]; - addr_to_page(addr, page); - - ptep = (pte_t *) &page[pte_index(va)]; - pte.pte = (ptep->pte & ~_PAGE_RW); - xen_l1_entry_update(ptep, pte); - __flush_tlb_one(addr); -} - -static void __make_page_writable(unsigned long va) -{ - unsigned long addr; - pte_t pte, *ptep; - unsigned long *page = (unsigned long *) init_level4_pgt; - - addr = (unsigned long) page[pgd_index(va)]; + addr = page[pmd_index(_va)]; addr_to_page(addr, page); - addr = page[pud_index(va)]; - addr_to_page(addr, page); - - addr = page[pmd_index(va)]; - addr_to_page(addr, page); + ptep = (pte_t *) &page[pte_index(_va)]; - ptep = (pte_t *) &page[pte_index(va)]; - pte.pte = (ptep->pte | _PAGE_RW); - xen_l1_entry_update(ptep, pte); - __flush_tlb_one(addr); + pte.pte = ptep->pte & ~_PAGE_RW; + if (HYPERVISOR_update_va_mapping(_va, pte, 0)) + BUG(); } - -/* - * Assume the translation is already established. - */ void make_page_readonly(void *va) { - pgd_t* pgd; pud_t *pud; pmd_t* pmd; pte_t pte, *ptep; + pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t pte, *ptep; unsigned long addr = (unsigned long) va; - if (!init_mapping_done) { - __make_page_readonly(addr); - return; - } - pgd = pgd_offset_k(addr); pud = pud_offset(pgd, addr); pmd = pmd_offset(pud, addr); ptep = pte_offset_kernel(pmd, addr); - pte.pte = (ptep->pte & ~_PAGE_RW); - xen_l1_entry_update(ptep, pte); - __flush_tlb_one(addr); + + pte.pte = ptep->pte & ~_PAGE_RW; + if (HYPERVISOR_update_va_mapping(addr, pte, 0)) + xen_l1_entry_update(ptep, pte); /* fallback */ + + if ((addr >= VMALLOC_START) && (addr < VMALLOC_END)) + make_page_readonly(__va(pte_pfn(pte) << PAGE_SHIFT)); } void make_page_writable(void *va) { - pgd_t* pgd; pud_t *pud; pmd_t* pmd; pte_t pte, *ptep; + pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t pte, *ptep; unsigned long addr = (unsigned long) va; - if (!init_mapping_done) { - __make_page_writable(addr); - return; - } - pgd = pgd_offset_k(addr); pud = pud_offset(pgd, addr); pmd = pmd_offset(pud, addr); ptep = pte_offset_kernel(pmd, addr); - pte.pte = (ptep->pte | _PAGE_RW); - xen_l1_entry_update(ptep, pte); - __flush_tlb_one(addr); + + pte.pte = ptep->pte | _PAGE_RW; + if (HYPERVISOR_update_va_mapping(addr, pte, 0)) + xen_l1_entry_update(ptep, pte); /* fallback */ + + if ((addr >= VMALLOC_START) && (addr < VMALLOC_END)) + make_page_writable(__va(pte_pfn(pte) << PAGE_SHIFT)); } -void make_pages_readonly(void* va, unsigned nr) +void make_pages_readonly(void *va, unsigned nr) { while (nr-- != 0) { make_page_readonly(va); @@ -160,7 +132,7 @@ void make_pages_readonly(void* va, unsigned nr) } } -void make_pages_writable(void* va, unsigned nr) +void make_pages_writable(void *va, unsigned nr) { while (nr-- != 0) { make_page_writable(va); @@ -457,7 +429,7 @@ static void __init phys_pud_init(pud_t *pud, unsigned long address, unsigned lon } pmd = alloc_static_page(&pmd_phys); - make_page_readonly(pmd); + early_make_page_readonly(pmd); xen_pmd_pin(pmd_phys); set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE)); @@ -487,7 +459,7 @@ static void __init phys_pud_init(pud_t *pud, unsigned long address, unsigned lon __set_pte(pte, __pte(paddr | _KERNPG_TABLE)); } pte = pte_save; - make_page_readonly(pte); + early_make_page_readonly(pte); xen_pte_pin(pte_phys); set_pmd(pmd, __pmd(pte_phys | _KERNPG_TABLE)); } @@ -536,11 +508,11 @@ void __init xen_init_pt(void) _KERNPG_TABLE | _PAGE_USER); memcpy((void *)level2_kernel_pgt, page, PAGE_SIZE); - make_page_readonly(init_level4_pgt); - make_page_readonly(init_level4_user_pgt); - make_page_readonly(level3_kernel_pgt); - make_page_readonly(level3_user_pgt); - make_page_readonly(level2_kernel_pgt); + early_make_page_readonly(init_level4_pgt); + early_make_page_readonly(init_level4_user_pgt); + early_make_page_readonly(level3_kernel_pgt); + early_make_page_readonly(level3_user_pgt); + early_make_page_readonly(level2_kernel_pgt); xen_pgd_pin(__pa_symbol(init_level4_pgt)); xen_pgd_pin(__pa_symbol(init_level4_user_pgt)); @@ -578,7 +550,7 @@ void __init extend_init_mapping(void) pmd = (pmd_t *)&page[pmd_index(va)]; if (pmd_none(*pmd)) { pte_page = alloc_static_page(&phys); - make_page_readonly(pte_page); + early_make_page_readonly(pte_page); xen_pte_pin(phys); set_pmd(pmd, __pmd(phys | _KERNPG_TABLE | _PAGE_USER)); } else { @@ -625,7 +597,7 @@ void __init init_memory_mapping(unsigned long start, unsigned long end) for (; start < end; start = next) { unsigned long pud_phys; pud_t *pud = alloc_static_page(&pud_phys); - make_page_readonly(pud); + early_make_page_readonly(pud); xen_pud_pin(pud_phys); next = start + PGDIR_SIZE; if (next > end) @@ -640,7 +612,6 @@ void __init init_memory_mapping(unsigned long start, unsigned long end) BUG_ON(start_pfn != (table_start + (tables_space >> PAGE_SHIFT))); __flush_tlb_all(); - init_mapping_done = 1; } extern struct x8664_pda cpu_pda[NR_CPUS]; -- 2.30.2